import requests
from pyquery import PyQuery as pq
from time import sleep
import random

files=open("ip.txt","a+",encoding="utf-8")

def checkIP(ip):
    '''检测免费代理是否可用'''
    print("检测ip:",ip)
    proxy = ip
    #需要认证的代理
    #proxy = 'username:password@127.0.0.1:8888'

    proxies = {
        'http': 'http://' + proxy,
        'https': 'https://' + proxy,
    }
    try:
        response = requests.get('http://httpbin.org/get', proxies=proxies)
        # files.write("'"+ip+"',\n") # ip可用 则存入文件
        response.close()
    except requests.exceptions.ConnectionError as e:
        print('Error', e.args)


def getIP():
    '''爬取免费代理IP地址'''

    proxy_list=[
    '115.236.190.204:80',
    '163.125.28.243:8118',
    '111.177.161.150:9999',
    '49.77.58.78:8118',
    '59.57.151.126:31689',
    '222.92.112.69:8080',
    '118.182.33.6:42801',
    '111.75.193.22:48449',
    '47.94.104.204:8118',
    '121.232.194.115:9999',
    '58.49.134.202:41521',
    '113.13.160.65:9999',
    '219.159.38.206:56210',
    '111.8.150.52:39315',
    '111.177.163.126:9999',
    '119.101.112.6:9999',
    '111.75.193.25:48449',
    '118.182.33.6:42801',
    '49.77.58.78:8118',
    '59.57.151.126:31689',
    '111.75.193.22:48449',
    ]

    for i in range(10,20):
        # 随机选择一个代理
        proxy =  random.choice(proxy_list)
        proxies = {
            'http': 'http://' + proxy,
            'https': 'https://' + proxy,
        }
        headers={
            'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)'
        }
        print("第几页：",i)
        try:
            url="https://www.kuaidaili.com/free/inha/"+str(i)
            res=requests.get(url,proxies=proxies,headers=headers)
            html=res.content.decode("utf-8")

            # 解析得到所有图书标签列表
            doc=pq(html)
            tagItems=doc("#list tr")
            # 循环将图书标签链接写入redis中
            for tr in tagItems.items():
                # 图书标签链接
                host=tr.find("td:nth-child(1)").text()
                port=tr.find("td:nth-child(2)").text()
                if host:
                    ip=host+":"+port
                    # print(ip)
                    checkIP(ip)
            res.close()
        except requests.exceptions.ConnectionError as e:
            print('Error', e.args)
        sleep(5)

if __name__=='__main__':
    # getIP()
    checkIP("123.160.17.33:30603")